# -*- coding: utf-8 -*-
import hashlib
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.utils.python import to_bytes

from steam_data_collect.models import rule_infos
from steam_data_collect.items import SteamDataCollectItem, imageitem
import logging

from steam_data_collect.settings import BASE_DIR
from scrapy.pipelines.images import ImagesPipeline
loger=logging.getLogger(__file__)

class SteamSpider(CrawlSpider):
    name = 'steam'
    allowed_domains = ['wikkistix.com']
    start_urls = ['https://www.wikkistix.com/stem-and-steam/']

    custom_settings = {'DEPTH_PRIORITY':1,
                        'DEPTH_STATS_VERBOSE': True,
                       'USER_AGENT':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
                       'COOKIES_ENABLED':False,
                       'DOWNLOAD_TIMEOUT':60,
                       }

    rules = (
        Rule(LinkExtractor(restrict_xpaths=("//a[@class='fusion-no-lightbox']/ancestor::div[@class='fusion-column-wrapper']/div[1]/div[1]/div[1]//a",)), callback='parse_item', follow=False),
    )


    def parse_item(self, response):
        loger.info(response.url)

        rule_list=rule_infos.select().where(rule_infos.url==response.url)
        if rule_list.count()>0:
            one_rule=rule_list[0]

            if one_rule.class_title!='' and one_rule.class_title!=None:
                class_title=response.xpath(one_rule.class_title).extract()
                # class_title=u''.join(class_title)
            else:
                class_title=[]
            if one_rule.class_desc!='' and one_rule.class_desc!=None:
                class_desc=response.xpath(one_rule.class_desc).extract()
                # class_desc=u''.join(class_desc)
            else:
                class_desc=[]
            if one_rule.class_img!='' and one_rule.class_img!=None:
                class_img=response.xpath(one_rule.class_img).extract()
                # class_img=u';'.join(class_img)
            else:
                class_img=[]
            if one_rule.challenge_title!='' and one_rule.challenge_title!=None:
                challenge_title=response.xpath(one_rule.challenge_title).extract()
                # challenge_title=u''.join(challenge_title)
            else:
                challenge_title=[]
            if one_rule.challenge_desc!='' and one_rule.challenge_desc!=None:
                challenge_desc=response.xpath(one_rule.challenge_desc).extract()
                # challenge_desc=u''.join(challenge_desc)
            else:
                challenge_desc=[]
            if one_rule.objective!='' and one_rule.objective!=None:
                objective=response.xpath(one_rule.objective).extract()
                # objective=u''.join(objective)
            else:
                objective=[]
            if one_rule.background_information!='' and one_rule.background_information!=None:
                background_information=response.xpath(one_rule.background_information).extract()
                # background_information=u''.join(background_information)
            else:
                background_information=[]
            if one_rule.science_skills!='' and one_rule.science_skills!=None:
                science_skills=response.xpath(one_rule.science_skills).extract()
                # science_skills=u''.join(science_skills)
            else:
                science_skills=[]
            if one_rule.technology_skills!='' and one_rule.technology_skills!=None:
                technology_skills=response.xpath(one_rule.technology_skills).extract()
                # technology_skills=u''.join(technology_skills)
            else:
                technology_skills=[]
            if one_rule.engineering_skills!='' and one_rule.engineering_skills!=None:
                engineering_skills=response.xpath(one_rule.engineering_skills).extract()
                # engineering_skills=u''.join(engineering_skills)
            else:
                engineering_skills=[]
            if one_rule.arts_skills!='' and one_rule.arts_skills!=None:
                arts_skills=response.xpath(one_rule.arts_skills).extract()
                # arts_skills=u''.join(arts_skills)
            else:
                arts_skills=[]
            if one_rule.math_skills!='' and one_rule.math_skills!=None:
                math_skills=response.xpath(one_rule.math_skills).extract()
                # math_skills=u''.join(math_skills)
            else:
                math_skills=[]
            if one_rule.directions!='' and one_rule.directions!=None:
                directions=response.xpath(one_rule.directions).extract()
                # directions=u''.join(directions)
            else:
                directions=[]
            if one_rule.materials!='' and one_rule.materials!=None:
                materials=response.xpath(one_rule.materials).extract()
                # materials=u''.join(materials)
            else:
                materials=[]
            if one_rule.wrap_up!='' and one_rule.wrap_up!=None:
                wrap_up=response.xpath(one_rule.wrap_up).extract()
                # wrap_up=u''.join(wrap_up)
            else:
                wrap_up=[]
            if one_rule.extended_picture!='' and one_rule.extended_picture!=None:
                extended_picture=response.xpath(one_rule.extended_picture).extract()
                # extended_picture=u';'.join(extended_picture)
            else:
                extended_picture=[]
            if one_rule.achievements_picture!='' and one_rule.achievements_picture!=None:
                achievements_picture=response.xpath(one_rule.achievements_picture).extract()
                # achievements_picture=u';'.join(achievements_picture)
            else:
                achievements_picture=[]

            item_obj=SteamDataCollectItem()

            item_obj['url']=response.url
            item_obj["class_title"]=class_title
            item_obj["class_desc"]=class_desc
            item_obj["class_img"]=class_img
            item_obj["challenge_title"]=challenge_title
            item_obj["challenge_desc"]=challenge_desc
            item_obj["objective"]=objective
            item_obj["background_information"]=background_information
            item_obj["science_skills"]=science_skills
            item_obj["technology_skills"]=technology_skills
            item_obj["engineering_skills"]=engineering_skills
            item_obj["arts_skills"]=arts_skills
            item_obj["math_skills"]=math_skills
            item_obj["directions"]=directions
            item_obj["materials"]=materials
            item_obj["wrap_up"]=wrap_up
            item_obj["extended_picture"]=extended_picture
            item_obj["achievements_picture"]=achievements_picture

            img_name_list =[]
            for sub in class_img:
                image_guid = hashlib.sha1(to_bytes(sub)).hexdigest()
                img_name_list.append('full/'+image_guid+'.jpg')
                image_obj = imageitem()
                image_obj['image_urls']=[sub]
                yield image_obj
            item_obj["class_img"] = img_name_list

            img_name_list = []
            for sub in extended_picture:
                image_guid = hashlib.sha1(to_bytes(sub)).hexdigest()
                img_name_list.append('full/'+image_guid + '.jpg')
                image_obj = imageitem()
                image_obj['image_urls'] = [sub]
                yield image_obj
            item_obj["extended_picture"] = img_name_list

            img_name_list = []
            for sub in achievements_picture:
                image_guid = hashlib.sha1(to_bytes(sub)).hexdigest()
                img_name_list.append('full/'+image_guid + '.jpg')
                image_obj = imageitem()
                image_obj['image_urls'] = [sub]
                yield image_obj
            item_obj["achievements_picture"] = img_name_list

            yield item_obj

